import MySQLdb ,json ,gzip
from collections import defaultdict, Counter

print "Load Dictionary"
with gzip.open("inverse_word_collection_frequencies.json.gz",'r') as orfile:
	IDF = Counter(json.loads(orfile.readline()))

stop = False
while not stop:
	print "Open Connection"
	ronn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
	conn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
	conn.autocommit(False)
	reader = ronn.cursor(MySQLdb.cursors.SSCursor)
	writer = conn.cursor()
	print "Ask for all Discriptions"
	got = reader.execute("""SELECT * FROM Books WHERE `Frequencies` = "" LIMIT 0, 500""")
	if got < 500: stop = True
	BUFFER = []
	row = reader.fetchone()
	iterator = 0
	# For every Description
	while row is not None:
		iterator += 1
		print "Reading Description", iterator
		Product = row[0]
		Dictionary = Counter(json.loads(row[1]))
		TFIDF = defaultdict(float)
		COUNT = float(sum(Dictionary.values()))
		# For Every Term calculate tf/idf
		for word in Dictionary:
			TFIDF[word] = ((Dictionary[word] / COUNT) * IDF[word])
		BUFFER.append((json.dumps(TFIDF), Product))
		# Store Object In Database
		if len(BUFFER)%500 == 0:
			print "Storing..."
			writer.executemany("""UPDATE Books SET Frequencies = %s WHERE ProductID = %s""",BUFFER)
			BUFFER = []
			print "Stored 500 Objects to database"
			conn.commit()
		row = reader.fetchone()
		
	writer.executemany("""UPDATE Books SET Frequencies = %s WHERE ProductID = %s""",BUFFER)
	conn.commit()
	reader.close()
	conn.close()
	ronn.close()


